

var Crawler = require('crawler');
var articles = require('./models/articles'); // 创建一个表

var c = new Crawler({
    maxConnections: 10,
    //这将为每个爬取的页面调用 
    callback: function (error, res, done) {
        if (error) {
            console.log(error);
        } else {
            var $ = res.$;

            var list = $('.dw_table .el').toArray();
            list.forEach(element => {
                var txt = $(element);
                var Jobtitle=txt.find('p.t1 span').text().trim();
                if(Jobtitle=='数据分析师'){
                    var Company=txt.find('.el span.t2 a').text();
                    var Workingplace=txt.find('.el span.t3').text();
                    var salary=txt.find('.el span.t4').text();
                    var Releasetime=txt.find('.el span.t5').text();
                    var time=new Date().getFullYear();
                    articles.create({
                        Jobtitle:Jobtitle,
                        Company:Company,
                        Workingplace:Workingplace,
                        salary:salary,
                        Releasetime:`${time}-${Releasetime}`
                    }).then((row)=>{
                        console.log('添加的Id：'+row.id);
                    })
                    // console.log(Workingplace);
                }

            });

        }
        done();
    }
});

// Queue just one URL, with default callback
// 队列只有一个url，默认回调 


module.exports = (url) => {
    c.queue(url)
}
